# -*- coding: utf-8 -*-
from scrapy.spider import CrawlSpider,Rule
from scrapy.linkextractors import LinkExtractor
from csdn.items import MasterItem


class CoursesSpider(CrawlSpider):
    name = 'master'
    allowed_domains = ['edu.csdn.net']
    start_urls = ['https://edu.csdn.net/courses/k/']
    item = MasterItem()

    #Rule是在定义抽取链接的规则
    rules = (  
        Rule(LinkExtractor(allow=(r'https://edu.csdn.net/courses/k/p[0-9]+',)), callback='parse_item', follow=True),  
    )  

    def parse_item(self, response):
        item = self.item
        courselist = response.selector.css("div.course_item")
        for course in courselist:
        	item['url'] = course.css("a::attr(href)").extract_first()
        	print(item)
        	yield item

